#===============================================================================
# 7_text_study_figure_10_appG.R
# Purpose: To replicate Figure 10 in Appendix G of the paper: 
# "Modeling IRR in the text study II"
#===============================================================================

# PACKAGES
#===============================================================================
library(dplyr)
library(tidyr)
library(ggplot2)
library(broom)
library(lme4)
library(xtable)

# DATA
#===============================================================================
# - load the pairwise dataset. N = 4,417
db <- read.csv("./data/text-experiment-merged-dataset-PAIRWISE-IRR.csv")

# - define references categories for the categorical variables
db <- db %>% 
  mutate(training = factor(training, levels = c("Basic", "Intermediate", 
                                                "Advanced")),
         task = relevel(factor(task), ref = "DIRECTED_AT")) %>%
  rename(sameideology = samgeideology) %>%
  na.omit() # dropping 2 obs. with NAs. N = 4,415

# MAIN
#===============================================================================

# - estimate models shown in Appendix G, Figure 10

# - Model 1: training
model01ml <- lmer(irr_cohen ~ training + (1 | pair),
                  data = db)

# - Model 2: task
model02ml <- lmer(irr_cohen ~ task + (1 | pair),
                  data = db)

# - Model 3: samegender
model03ml <- lmer(irr_cohen ~ samegender + (1 | pair),
                  data = db)

# - Model 4: samegender * task
model04ml <- lmer(irr_cohen ~ samegender * task + (1 | pair),
                  data = db)

# - Model 5: samegender *  training
model05ml <- lmer(irr_cohen ~ samegender * training + (1 | pair),
                  data = db)

# - Model 6: samegender * task * training
model06ml <- lmer(irr_cohen ~ samegender * task * training + (1 | pair),
                  data = db)

# - Model 7: sameideology
model07ml <- lmer(irr_cohen ~ sameideology + (1 | pair),
                  data = db)

# - Model 8: sameideology * tasks
model08ml <- lmer(irr_cohen ~ sameideology * task + (1 | pair),
                  data = db)

# - Model 9: sameideology * training
model09ml <- lmer(irr_cohen ~ sameideology * training + (1 | pair),
                  data = db)

# - Model 10: sameideology * task * training
model10ml <- lmer(irr_cohen ~ sameideology * task * training + (1 | pair),
                  data = db)

# - Model 11: sameideology + samegender
model11ml <- lmer(irr_cohen ~ sameideology + samegender + (1 | pair),
                  data = db)

# - COMBINE all the model results
allmodels <- rbind(
  # Model 1
  as.data.frame(summary(model01ml)$coefficients) %>% 
    mutate(term = rownames(summary(model01ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 1"),
  # Model 2
  as.data.frame(summary(model02ml)$coefficients) %>% 
    mutate(term = rownames(summary(model02ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 2"),
  # Model 3
  as.data.frame(summary(model03ml)$coefficients) %>% 
    mutate(term = rownames(summary(model03ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 3"),
  # Model 4
  as.data.frame(summary(model04ml)$coefficients) %>% 
    mutate(term = rownames(summary(model04ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 4"),
  # Model 5
  as.data.frame(summary(model05ml)$coefficients) %>% 
    mutate(term = rownames(summary(model05ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 5"),
  # Model 6
  as.data.frame(summary(model06ml)$coefficients) %>% 
    mutate(term = rownames(summary(model06ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 6"),
  # Model 7
  as.data.frame(summary(model07ml)$coefficients) %>% 
    mutate(term = rownames(summary(model07ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 7"),
  # Model 8
  as.data.frame(summary(model08ml)$coefficients) %>% 
    mutate(term = rownames(summary(model08ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 8"),
  # Model 9
  as.data.frame(summary(model09ml)$coefficients) %>% 
    mutate(term = rownames(summary(model09ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 9"),
  # Model 10
  as.data.frame(summary(model10ml)$coefficients) %>% 
    mutate(term = rownames(summary(model10ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 10"),
  # Model 11
  as.data.frame(summary(model11ml)$coefficients) %>% 
    mutate(term = rownames(summary(model11ml)$coefficients)) %>%
    rename(estimate = Estimate, std.error = `Std. Error`, statistic = `t value`) %>%
    mutate(model = "Model 11")
)

plotdb <- allmodels %>%
  filter(term != "(Intercept)") %>%
  mutate(term = recode(term, 
                       # - single predictors
                       `samegender` = "Same gender",
                       `sameideology` = "Same ideology",
                       `taskANGRY_FEEL` = "[Task] Feeling angry (ref: Directed at)",
                       `taskENTHU_FEEL` = "[Task] Feeling enthusiastic (ref: Directed at)",
                       `taskCONS_VIEW` = "[Task] Conservative view (ref: Directed at)",
                       `taskPROGR_VIEW` = "[Task] Progressive view (ref: Directed at)",
                       `taskNEG_TONE` = "[Task] Negative tone (ref: Directed at)",
                       `taskGENDER` = "[Task] Gender issue (ref: Directed at)",
                       `trainingIntermediate` = "[Training] Intermediate (ref: Basic)",
                       `trainingAdvanced` = "[Training] Advanced (ref: Basic)",
                       # - double interactions
                       `samegender:trainingIntermediate` = "(Same gender) X (Training) Intermediate (Ref:Basic)",
                       `samegender:trainingAdvanced` = "(Same gender) X (Training) Advanced (Ref:Basic)",
                       `samegender:taskANGRY_FEEL` = "(Same gender) X (Task) Feeling angry (Ref: Directed at)",
                       `samegender:taskCONS_VIEW` = "(Same gender) X (Task) Conservative view (Ref: Directed at)",
                       `samegender:taskENTHU_FEEL` = "(Same gender) X (Task) Feeling enthusiastic (Ref: Directed at)",
                       `samegender:taskGENDER` = "(Same gender) X (Task) Gender issue (Ref: Directed at)",
                       `samegender:taskNEG_TONE` = "(Same gender) X (Task) Negative tone (Ref: Directed at)",
                       `samegender:taskPROGR_VIEW` = "(Same gender) X (Task) Progressive view (Ref: Directed at)",
                       `sameideology:trainingIntermediate` = "(Same ideology) X (Training) Intermediate (Ref: Basic)",
                       `sameideology:trainingAdvanced` = "(Same ideology) X (Training) Advanced (Ref: Basic)",
                       `sameideology:taskANGRY_FEEL` = "(Same ideology) X (Task) Feeling angry (Ref: Directed at)",
                       `sameideology:taskCONS_VIEW` = "(Same ideology) X (Task) Conservative view (Ref: Directed at)",
                       `sameideology:taskENTHU_FEEL` = "(Same ideology) X (Task) Feeling enthusiastic (Ref: Directed at)",
                       `sameideology:taskGENDER` = "(Same ideology) X (Task) Gender issue (Ref: Directed at)",
                       `sameideology:taskNEG_TONE` = "(Same ideology) X (Task) Negative tone (Ref: Directed at)",
                       `sameideology:taskPROGR_VIEW` = "(Same ideology) X (Task) Progressive view (Ref: Directed at)",
                       # - additional double interactions`
                       `taskANGRY_FEEL:trainingIntermediate` = "(Task: Feel angry) X (Training: Intermediate)",
                       `taskCONS_VIEW:trainingIntermediate` = "(Task: Cons. view) X (Training: Intermediate)",
                       `taskENTHU_FEEL:trainingIntermediate` = "(Task: Feel enth.) X (Training: Intermediate)",
                       `taskGENDER:trainingIntermediate` = "(Task: Gender issue) X (Training: Intermediate)",
                       `taskNEG_TONE:trainingIntermediate` = "(Task: Neg. tone) X (Training: Intermediate)",
                       `taskPROGR_VIEW:trainingIntermediate` = "(Task: Progr. view) X (Training: Intermediate)",
                       `taskANGRY_FEEL:trainingAdvanced` = "(Task: Feel angry) X (Training: Advanced)",
                       `taskCONS_VIEW:trainingAdvanced` = "(Task: Cons. view) X (Training: Advanced)",
                       `taskENTHU_FEEL:trainingAdvanced` = "(Task: Feel enth.) X (Training: Advanced)",
                       `taskGENDER:trainingAdvanced` = "(Task: Gender issue) X (Training: Advanced)",
                       `taskNEG_TONE:trainingAdvanced` = "(Task: Neg. tone) X (Training: Advanced)",
                       `taskPROGR_VIEW:trainingAdvanced` = "(Task: Progr. view) X (Training: Advanced)",
                       # - triple interactions
                       `samegender:taskANGRY_FEEL:trainingIntermediate` = "(Same gender) X (Task: Feel angry) X (Training: Inter.)",
                       `samegender:taskCONS_VIEW:trainingIntermediate` = "(Same gender) X (Task: Cons. view) X (Training: Inter.)",
                       `samegender:taskENTHU_FEEL:trainingIntermediate` = "(Same gender) X (Task: Feel enth.) X (Training: Inter.)",
                       `samegender:taskGENDER:trainingIntermediate` = "(Same gender) X (Task: Gender issue) X (Training: Inter.)",
                       `samegender:taskNEG_TONE:trainingIntermediate` = "(Same gender) X (Task: Neg. tone) X (Training: Inter.)",
                       `samegender:taskPROGR_VIEW:trainingIntermediate` = "(Same gender) X (Task: Progr. view) X (Training: Inter.)",
                       `samegender:taskANGRY_FEEL:trainingAdvanced` = "(Same gender) X (Task: Feel angry) X (Training: Adv.)",
                       `samegender:taskCONS_VIEW:trainingAdvanced` = "(Same gender) X (Task: Cons. view) X (Training: Adv.)",
                       `samegender:taskENTHU_FEEL:trainingAdvanced` = "(Same gender) X (Task: Feel enth.) X (Training: Adv.)",
                       `samegender:taskGENDER:trainingAdvanced` = "(Same gender) X (Task: Gender issue) X (Training: Adv.)",
                       `samegender:taskNEG_TONE:trainingAdvanced` = "(Same gender) X (Task: Neg. tone) X (Training: Adv.)",
                       `samegender:taskPROGR_VIEW:trainingAdvanced` = "(Same gender) X (Task: Progr. view) X (Training: Adv.)",
                       `sameideology:taskANGRY_FEEL:trainingIntermediate` = "(Same ideology) X (Task: Feel angry) X (Training: Inter.)",
                       `sameideology:taskCONS_VIEW:trainingIntermediate` = "(Same ideology) X (Task: Cons. view) X (Training: Inter.)",
                       `sameideology:taskENTHU_FEEL:trainingIntermediate` = "(Same ideology) X (Task: Feel enth.) X (Training: Inter.)",
                       `sameideology:taskGENDER:trainingIntermediate` = "(Same ideology) X (Task: Gender issue) X (Training: Inter.)",
                       `sameideology:taskNEG_TONE:trainingIntermediate` = "(Same ideology) X (Task: Neg. tone) X (Training: Inter.)",
                       `sameideology:taskPROGR_VIEW:trainingIntermediate` = "(Same ideology) X (Task: Progr. view) X (Training: Inter.)",
                       `sameideology:taskANGRY_FEEL:trainingAdvanced` = "(Same ideology) X (Task: Feel angry) X (Training: Adv.)",
                       `sameideology:taskCONS_VIEW:trainingAdvanced` = "(Same ideology) X (Task: Cons. view) X (Training: Adv.)",
                       `sameideology:taskENTHU_FEEL:trainingAdvanced` = "(Same ideology) X (Task: Feel enth.) X (Training: Adv.)",
                       `sameideology:taskGENDER:trainingAdvanced` = "(Same ideology) X (Task: Gender issue) X (Training: Adv.)",
                       `sameideology:taskNEG_TONE:trainingAdvanced` = "(Same ideology) X (Task: Neg. tone) X (Training: Adv.)",
                       `sameideology:taskPROGR_VIEW:trainingAdvanced` = "(Same ideology) X (Task: Progr. view) X (Training: Adv.)"
  ),
  term = factor(term, levels = rev(c(
    # - SINGLE PREDICTORS
    "Same gender",
    "Same ideology",
    "[Task] Gender issue (ref: Directed at)",
    "[Task] Negative tone (ref: Directed at)",
    "[Task] Progressive view (ref: Directed at)",
    "[Task] Conservative view (ref: Directed at)",
    "[Task] Feeling angry (ref: Directed at)",
    "[Task] Feeling enthusiastic (ref: Directed at)",
    "[Training] Intermediate (ref: Basic)",
    "[Training] Advanced (ref: Basic)",
    # - DOUBLE INTERACTIONS
    "(Same gender) X (Training) Intermediate (Ref:Basic)",
    "(Same gender) X (Training) Advanced (Ref:Basic)",
    "(Same gender) X (Task) Gender issue (Ref: Directed at)",
    "(Same gender) X (Task) Negative tone (Ref: Directed at)",
    "(Same gender) X (Task) Progressive view (Ref: Directed at)",
    "(Same gender) X (Task) Conservative view (Ref: Directed at)",
    "(Same gender) X (Task) Feeling angry (Ref: Directed at)",
    "(Same gender) X (Task) Feeling enthusiastic (Ref: Directed at)",
    "(Same ideology) X (Training) Intermediate (Ref: Basic)",
    "(Same ideology) X (Training) Advanced (Ref: Basic)",
    "(Same ideology) X (Task) Gender issue (Ref: Directed at)",
    "(Same ideology) X (Task) Negative tone (Ref: Directed at)",
    "(Same ideology) X (Task) Progressive view (Ref: Directed at)",
    "(Same ideology) X (Task) Conservative view (Ref: Directed at)",
    "(Same ideology) X (Task) Feeling angry (Ref: Directed at)",
    "(Same ideology) X (Task) Feeling enthusiastic (Ref: Directed at)",
    # - ADDITIONAL DOUBLE INTERACTIONS
    "(Task: Feel angry) X (Training: Intermediate)",
    "(Task: Cons. view) X (Training: Intermediate)",
    "(Task: Feel enth.) X (Training: Intermediate)",
    "(Task: Gender issue) X (Training: Intermediate)",
    "(Task: Neg. tone) X (Training: Intermediate)",
    "(Task: Progr. view) X (Training: Intermediate)",
    "(Task: Feel angry) X (Training: Advanced)",
    "(Task: Cons. view) X (Training: Advanced)",
    "(Task: Feel enth.) X (Training: Advanced)",
    "(Task: Gender issue) X (Training: Advanced)",
    "(Task: Neg. tone) X (Training: Advanced)",
    "(Task: Progr. view) X (Training: Advanced)",
    # - TRIPLE INTERACTIONS
    "(Same gender) X (Task: Feel angry) X (Training: Inter.)",
    "(Same gender) X (Task: Cons. view) X (Training: Inter.)",
    "(Same gender) X (Task: Feel enth.) X (Training: Inter.)",
    "(Same gender) X (Task: Gender issue) X (Training: Inter.)",
    "(Same gender) X (Task: Neg. tone) X (Training: Inter.)",
    "(Same gender) X (Task: Progr. view) X (Training: Inter.)",
    "(Same gender) X (Task: Feel angry) X (Training: Adv.)",
    "(Same gender) X (Task: Cons. view) X (Training: Adv.)",
    "(Same gender) X (Task: Feel enth.) X (Training: Adv.)",
    "(Same gender) X (Task: Gender issue) X (Training: Adv.)",
    "(Same gender) X (Task: Neg. tone) X (Training: Adv.)",
    "(Same gender) X (Task: Progr. view) X (Training: Adv.)",
    "(Same ideology) X (Task: Feel angry) X (Training: Inter.)",
    "(Same ideology) X (Task: Cons. view) X (Training: Inter.)",
    "(Same ideology) X (Task: Feel enth.) X (Training: Inter.)",
    "(Same ideology) X (Task: Gender issue) X (Training: Inter.)",
    "(Same ideology) X (Task: Neg. tone) X (Training: Inter.)",
    "(Same ideology) X (Task: Progr. view) X (Training: Inter.)",
    "(Same ideology) X (Task: Feel angry) X (Training: Adv.)",
    "(Same ideology) X (Task: Cons. view) X (Training: Adv.)",
    "(Same ideology) X (Task: Feel enth.) X (Training: Adv.)",
    "(Same ideology) X (Task: Gender issue) X (Training: Adv.)",
    "(Same ideology) X (Task: Neg. tone) X (Training: Adv.)",
    "(Same ideology) X (Task: Progr. view) X (Training: Adv.)"
  ))),
  lwr = estimate - (1.96 * std.error),
  upr = estimate + (1.96 * std.error)) %>%
  rename(pe = estimate) %>%
  mutate(model = factor(model, levels = unique(model)))

# - plot
ggplot(plotdb,
       aes(x = term, y = pe, ymin = lwr, ymax = upr)) +
  geom_segment(aes(x = term, xend = term, y = lwr, yend = upr), size = 3,
               color = "gray60") +
  geom_point(aes(x = term, y = pe), size = 1, color = "black") +
  geom_hline(yintercept = 0, color = "red") +
  facet_wrap(~ model, nrow = 1) +
  #scale_x_continuous("",
  #                   breaks = seq(1, 26, 1), labels = levels(plotdb03$term)) +
  scale_y_continuous("\nCoefficient estimates\n(models predicting IRR for all unique pairs of coders)") +
  scale_x_discrete("") +
  coord_flip() +
  theme(panel.background = element_blank(),
        panel.grid.major = element_line(linetype = "dotted", color = "gray80"),
        strip.text = element_text(size = 12),
        axis.text.x = element_text(size = 9),
        axis.text.y = element_text(size = 12))

ggsave("./results/appendix_g/appG_10_linear_IRR_preds.pdf",
       width = 55, height = 37.5, units = "cm", dpi = 300)


# - COEFFICIENT TABLE FOR THE MODELS
coef_table <- plotdb %>%
  mutate(estimate = paste0(round(pe, 3), " (", round(std.error, 3), ")",
                           ifelse(abs(statistic) > 1.96, "*", " ")),
         estimate = ifelse(is.na(estimate), "", estimate)) %>%
  dplyr::select(term, estimate, model) %>%
  spread(model, estimate) %>%
  mutate(term = factor(term, levels = levels(plotdb$term))) %>%
  arrange(desc(term))

coef_table[is.na(coef_table)] <- "--"

### This is Appendix G, Table 8
# ... first 7 models
print(xtable(coef_table[,c(1:8)]), include.rownames = FALSE)

# ... remaining models
print(xtable(coef_table[,c(1, c(9:12))]), include.rownames = FALSE)
